from Factors import *
from Factors import CheckFactor
import Analysis
from Analysis import LearningFramework
import Core.IO as IO

def DaysSalesInReceivables_Index(dfFundamental, factorName, params):
    #
    # type = "_" + params["Type"]
    #
    dfFundamental["SalesInReceivables"] = dfFundamental["Receivable"] / dfFundamental["Sales"]

    # 基于4季报，回看1年
    params2 = {"Period": 4, "Year": 1, "GrowthType": "Index"}
    params2["Fields"] = ["SalesInReceivables"]
    CAGRGrowthCallback(dfFundamental, factorName, params2)

    #
    # print(dfFundamental)
    pass


def GrossMargin_Index(dfFundamental, factorName, params):
    #
    # type = "_" + params["Type"]
    #
    dfFundamental["GrossMargin"] = dfFundamental["GrossProfit"] / dfFundamental["Sales"]

    # 基于4季报，回看1年
    params2 = {"Period": 4, "Year": 1, "GrowthType": "Index"}
    params2["Fields"] = ["GrossMargin"]
    CAGRGrowthCallback(dfFundamental, factorName, params2)

    #
    # print(dfFundamental)
    pass


def AssetQuality_Index(dfFundamental, factorName, params):
    #
    # type = "_" + params["Type"]
    #
    dfFundamental["AssetQuality"] = 1 - ((dfFundamental["CurrentAsset"] + dfFundamental["PPE"]) / dfFundamental["TotalAsset"])

    # 基于4季报，回看1年
    params2 = {"Period": 4, "Year": 1, "GrowthType": "Index"}
    params2["Fields"] = ["AssetQuality"]
    CAGRGrowthCallback(dfFundamental, factorName, params2)

    #
    # print(dfFundamental)
    pass


def Depreciation_Index(dfFundamental, factorName, params):
    #
    # type = "_" + params["Type"]
    #
    dfFundamental["DRatio"] = dfFundamental["DepAmort"] / (dfFundamental["DepAmort"] + dfFundamental["PPE"])

    # 基于4季报，回看1年
    params2 = {"Period": 4, "Year": 1, "GrowthType": "Index"}
    params2["Fields"] = ["DRatio"]
    CAGRGrowthCallback(dfFundamental, factorName, params2)
    #
    # print(dfFundamental)
    pass


def SGA_Index(dfFundamental, factorName, params):
    #
    # type = "_" + params["Type"]
    #
    dfFundamental["SGARatio"] = dfFundamental["SGA"] / dfFundamental["Sales"]

    # 基于4季报，回看1年
    params2 = {"Period": 4, "Year": 1, "GrowthType": "Index"}
    params2["Fields"] = ["SGARatio"]
    CAGRGrowthCallback(dfFundamental, factorName, params2)
    #
    # print(dfFundamental)
    pass


def Leverage_Index(dfFundamental, factorName, params):
    #
    # type = "_" + params["Type"]
    #
    dfFundamental["Leverage"] = (dfFundamental["CurrentLiab"] + dfFundamental["LTDebt"]) / dfFundamental["TotalAsset"]

    # 基于4季报，回看1年
    params2 = {"Period": 4, "Year": 1, "GrowthType": "Index"}
    params2["Fields"] = ["Leverage"]
    CAGRGrowthCallback(dfFundamental, factorName, params2)
    #
    # print(dfFundamental)
    pass


def AccrualsToAsset(dfFundamental, factorName, params):
    #
    type = "_" + params["Type"]
    #
    dfFundamental[factorName] = (dfFundamental["NetIncome2" + type] - dfFundamental["Extraordinary" + type] - dfFundamental["OperatingCashFlow" + type]) / \
                                dfFundamental["TotalAsset" + type]
    #
    # print(dfFundamental)
    pass


def CalcMSCoreFactor(database, instruments, datetime1, datetime2, update=False):

    type = "LYR"
    CalcFundamentalFactor(database, instruments, datetime1, datetime2,
                          factorName="DaysSaleRecv_Index",
                          fields=["Sales", "Receivable"],
                          # quarterlyFields=["Sales", "Receivables"],
                          callback=DaysSalesInReceivables_Index,
                          # startIndex=72
                          # params={"Type": "LYR"}
                          )

    CalcFundamentalFactor(database, instruments, datetime1, datetime2,
                          factorName="GrossMargin_Index",
                          fields=["Sales", "COGS", "GrossProfit"],
                          callback=GrossMargin_Index,
                          # startIndex=72
                          )

    # Factor3 Asset Quality
    CalcFundamentalFactor(database, instruments, datetime1, datetime2,
                          factorName="AssetQuality_Index",
                          fields=["CurrentAsset", "PPE", "TotalAsset"],
                          callback=AssetQuality_Index,
                          # startIndex=72
                          )

    # Factor4 Sales Growth
    CalcFundamentalFactor(database, instruments, datetime1, datetime2,
                          factorName="Growth_CAGR_Sales_1Yr",
                          callback=CAGRGrowthCallback,
                          fields=["Sales"],
                          params={"Period": 4, "Year": 1, "GrowthType": "CAGR"},
                          outputField=["Remark"],
                          update=update, test=False)

    # Factor5 Accruals
    CalcFundamentalFactor(database, instruments, datetime1, datetime2,
                          factorName="AccrualsToTotalAsset" + "_" + type,
                          fields=["NetIncome2", "Extraordinary", "OperatingCashFlow", "TotalAsset"],
                          callback=AccrualsToAsset,
                          params={"Type": type}
                          # startIndex=72
                          )

    # Factor6 Depreciation
    CalcFundamentalFactor(database, instruments, datetime1, datetime2,
                          factorName="Depreciation_Index",
                          fields=["DepAmort", "PPE"],
                          callback=Depreciation_Index,
                          # startIndex=72
                          )

    # Factor 7 SGA Index
    CalcFundamentalFactor(database, instruments, datetime1, datetime2,
                          factorName="SGA_Index",
                          fields=["SGA", "Sales"],
                          callback=SGA_Index,
                          # startIndex=72
                          )

    # Factor 8 Leverage Index
    CalcFundamentalFactor(database, instruments, datetime1, datetime2,
                          factorName="Leverage_Index",
                          fields=["CurrentLiab", "LTDebt", "TotalAsset"],
                          callback=Leverage_Index,
                          # startIndex=72
                          )


def CheckFactors(database):

    factorName = "DaysSaleRecv_Index"
    factorName = "GrossMargin_Index"
    factorName = "AssetQuality_Index"
    factorName = "Growth_CAGR_Sales_1Yr"
    factorName = "AccrualsToTotalAsset_LYR"
    factorName = "Depreciation_Index"
    factorName = "SGA_Index"
    factorName = "Leverage_Index"

    #
    CheckFactor.PlotFactor(database, factorName)
    #

    # ---某个时间点---
    datetime1 = datetime.datetime(2017, 12, 31)
    df = CheckFactor.PlotFactorProfile(database, factorName, datetime1)

    datetime1 = datetime.datetime(2000, 1, 1)
    datetime2 = datetime.datetime(2019, 6, 1)
    CheckFactor.PlotFactorProfileHistory2(database, datetime1, datetime2, factorName, dateTimeField="ReportDate")

    pass


def CacheData(database, datetime1):
    #
    factors = Factors()

    # ---Save to Cache---
    for factorName, value in factors.items():
        df = Analysis.General.Profile2(database, [factorName], datetime1, datetimeField="ReportDate")
        if factorName == "Growth_CAGR_Sales_1Yr":
            df[factorName] = df[factorName] + 1

        df.set_index("Symbol", inplace=True)
        print(df.describe())
        # Analysis.PreProcess.Outlier_Percentile(df)
        # print(df.describe())
        factors[factorName] = df
        # print(df)
        pass

    #
    return factors


def Factors():
    factors = {}
    factors["DaysSaleRecv_Index"] = None
    factors["GrossMargin_Index"] = None
    factors["AssetQuality_Index"] = None
    factors["Growth_CAGR_Sales_1Yr"] = None
    factors["AccrualsToTotalAsset_LYR"] = None
    factors["Depreciation_Index"] = None
    factors["SGA_Index"] = None
    factors["Leverage_Index"] = None
    return factors

#
def AllignFactors(database, datetime1):
    #
    factors = CacheData(database, datetime1)
    #
    instruments = Gadget.FindListedInstrument(database, datetime1)
    data = []
    for instrument in instruments:
        symbol = instrument["Symbol"]
        values = []
        complete = True
        for factorName in factors.keys():
            df = factors[factorName]
            if symbol not in df.index:
                print(symbol, "MissingFactor", factorName)
                complete = False
                break
            value = df.loc[symbol, factorName]
            values.append(value)
            pass

        if complete:
            # print(symbol, "MScore", mScore)
            data.append([symbol, datetime1.date()] + values)

    #
    df_MScore_Factors = pd.DataFrame(data, columns=["Symbol", "DateTime"] + list(factors.keys()))
    return df_MScore_Factors


def Coef_Classical():
    coef = {}
    coef["Intercept"] = -4.84
    coef["DaysSaleRecv_Index"] = 0.92
    coef["GrossMargin_Index"] = 0.528
    coef["AssetQuality_Index"] = 0.404
    coef["Growth_CAGR_Sales_1Yr"] = 0.892
    coef["Depreciation_Index"] = 0.115
    coef["SGA_Index"] = -0.172
    coef["AccrualsToTotalAsset_LYR"] = 4.679
    coef["Leverage_Index"] = -0.327
    #
    return coef

def Coef_Experience():
    coef = {}
    coef["Intercept"] = 0
    coef["DaysSaleRecv_Index"] = 0.1
    coef["GrossMargin_Index"] = 0.1
    coef["AssetQuality_Index"] = 0.1
    coef["Growth_CAGR_Sales_1Yr"] = 0.1
    coef["Depreciation_Index"] = -0.1
    coef["SGA_Index"] = -0.1
    coef["AccrualsToTotalAsset_LYR"] = 0.2
    coef["Leverage_Index"] = -0.1
    #
    return coef


# 基于传统系数计算，个别项目倒数处理
def RunModel(database, datetime1, datetime2):
    #
    # ---Save to Cache---
    factors = CacheData(database, datetime1)

    #
    instruments = Gadget.FindListedInstrument(database, datetime1)
    data = []
    for instrument in instruments:
        symbol = instrument["Symbol"]

        #
        coef = Coef_Classical()
        mScore = coef["Intercept"]
        complete = True
        for factorName in factors.keys():
            df = factors[factorName]
            if symbol not in df.index:
                print(symbol, "MissingFactor", factorName)
                complete = False
                break
            value = df.loc[symbol, factorName]
            if factorName == "GrossMargin_Index" or factorName == "Depreciation_Index":
               if value != 0:
                   value = 1/value
            mScore += coef[factorName] * value
            pass

        if complete:
            # print(symbol, "MScore", mScore)
            data.append([symbol, mScore, datetime1.date()])

    #
    df_MScore = pd.DataFrame(data, columns=["Symbol", "MScore", "DateTime"])
    return df_MScore


# 基于经验值系数计算，不做倒数处理
def RunModel2(database, datetime1, datetime2):
    #
    factors = CacheData(database, datetime1)

    # ProcessFactors
    for factorName, df in factors.items():
        # print(factorName)
        # print(df[factorName].describe())
        # 如果坚持不做标准化的话（这样不必等到5月1日），注意要把所有因子统一到一个比例维度上
        if "Accruals" in factorName:
            pass
        elif "Depreciation" in factorName:
            df = df[df[factorName] != 0]
        elif "GrossMargin" in factorName:
            df = df[df[factorName] != 0]
        else:
            pass
        # print(df[factorName].describe())
        # df.to_csv("d://" + factorName + "1.csv")
        Analysis.PreProcess.Outlier_Percentile(df)
        # df.to_csv("d://" + factorName + "2.csv")
        Analysis.PreProcess.Normalization(df)
        # df.to_csv("d://" + factorName + "3.csv")
        # print(df[factorName].describe())
        factors[factorName] = df
        pass

    # 基于经验值系数计算
    coef = Coef_Experience()

    #
    instruments = Gadget.FindListedInstrument(database, datetime1)
    data = []
    for instrument in instruments:
        symbol = instrument["Symbol"]
        #
        if symbol == "000037.SZ":
            pass
        #
        mScore = coef["Intercept"]
        complete = True
        missing = 0
        factorValues = []
        for factorName in factors.keys():
            df = factors[factorName]
            if symbol not in df.index:
                # print(symbol, "MissingFactor", factorName)
                missing += 1
                factorValues.append(0)
                continue

            value = df.loc[symbol, factorName]
            factorValues.append(value)
            mScore += coef[factorName] * value
            pass

        if missing <= 2: # 缺少少数因子Ok
            # print(symbol, "MScore", mScore)
            data.append([symbol, datetime1.date()] + factorValues + [mScore])
    #

    df_MScore = pd.DataFrame(data, columns=["Symbol", "DateTime"] + list(factors.keys()) + ["MScore"])
    return df_MScore


def ExportMScore(database):
    for year in range(2006,2020):
        #
        datetime1 = datetime.datetime(year,12,31)
        #
        df_MScore = RunModel(database, datetime1, None)
        #
        datetime2 = datetime.datetime(year+1,5,1)
        df_MScore.to_csv("d://Data//Profile//MScore//Profile_" + Gadget.ToDateString(datetime2) + ".csv")


def ExportMScoreFactors(database):
    for year in range(2006,2020):
        #
        datetime1 = datetime.datetime(year,12,31)
        df_MScore = AllignFactors(database, datetime1)
        #
        datetime2 = datetime.datetime(year+1,5,1)
        df_MScore.to_csv("d://Data//Profile//MScoreFactor//Profile_" + Gadget.ToDateString(datetime2) + ".csv")


def SaveToDatabase(database):
    #
    for year in range(2018,2019):
        #
        datetime1 = datetime.datetime(year,12,31)
        df_MScore = RunModel2(database, datetime1, None)
        #
        datetime2 = datetime.datetime(year+1,5,1)
        df_MScore.to_csv("d://Data//Profile//MScore//MScore_NormFactor_Experience_Profile_" + Gadget.ToDateString(datetime2) + ".csv")

        data = []
        for index, row in df_MScore.iterrows():
            doc = {}
            doc["Key2"] = row["Symbol"] + "_" + Gadget.ToDateString(datetime2)
            doc["DateTime"] = datetime2
            doc["Date"] = datetime2
            doc["ReportDate"] = datetime1
            doc["UTCDateTime"] = Gadget.ToUTCDateTime(datetime2)
            doc["Symbol"] = row["Symbol"]
            doc["Value"] = row["MScore"]
            doc["Period"] = 4
            data.append(doc)
        #
        database.Upsert_Many("Factor", "MScore_Experience", {}, data)

        pass


def TestModel(database):

    datetime1 = datetime.datetime(2007, 1, 1)
    datetime2 = datetime.datetime(2019, 6, 1)
    params = {}
    params["KeepRedundant"] = True
    # params = {"ThresholdYData": -0.3}
    df = Analysis.LearningFramework.MultiPeriodData(database, datetime1, datetime2,
                                                    factors=["Score","MScore"],
                                                    profileFolderName="MScore",
                                                    rangeReturnFolderName="Yearly",
                                                    transformDataPrarams=params
                                                    )

    def function(x, gtlt, threshold):
        if gtlt == ">":
            if x > threshold:
                return 1
            else:
                return 0
        elif gtlt == "<":
            if x < threshold:
                return 1
            else:
                return 0
        else:
            return 0

    # ---利用经验值手工测试---
    df["Y_True"] = 0
    df["Y_Predict"] = 0

    ret = -40

    print("Scenerio1")
    df["Y_True"]    = df.apply(lambda x: function(x["ExcessReturn"], "<", ret), axis=1)
    df["Y_Predict"] = df.apply(lambda x: function(x["Score"], ">", -2), axis=1)
    Analysis.LearningFramework.Performance_Metric(df["Y_True"], df["Y_Predict"])

    print("Scenerio2")
    df["Y_True"]    = df.apply(lambda x: function(x["ExcessReturn"], "<", ret), axis=1)
    df["Y_Predict"] = df.apply(lambda x: function(x["Score"], ">", -1.5), axis=1)
    Analysis.LearningFramework.Performance_Metric(df["Y_True"], df["Y_Predict"])

    print("Scenerio3")
    df["Y_True"]    = df.apply(lambda x: function(x["ExcessReturn"], "<", ret), axis=1)
    df["Y_Predict"] = df.apply(lambda x: function(x["Score"], ">", -1), axis=1)
    Analysis.LearningFramework.Performance_Metric(df["Y_True"], df["Y_Predict"])

    print("Scenerio4")
    df["Y_True"]    = df.apply(lambda x: function(x["ExcessReturn"], "<", ret), axis=1)
    df["Y_Predict"] = df.apply(lambda x: function(x["Score"], ">", 0), axis=1)
    Analysis.LearningFramework.Performance_Metric(df["Y_True"], df["Y_Predict"])

    df.to_csv("d:/MScoreMerged.csv")
    pass


def TrainModel(database):
    #
    #
    instruments = database.Find("Instruments", "Stock" , {"Industry":"有色金属"})
    data = []
    for instrument in instruments:
        data.append(instrument["Symbol"])
    # print(dfSymbols)

    #
    datetime1 = datetime.datetime(2007, 1, 1)
    datetime2 = datetime.datetime(2019, 6, 1)

    #
    params = {}
    params["DropRedundantFields"] = True
    # params["KeepFields"] = ["Symbol", "DateTime", "Return", "ExcessReturn"]
    # params["KeepFields"] = ["Symbol"]
    params["ProcessY_Threshold"] = -0.1
    params["ProcessY_RiskAlpha"] = "Risk"
    params["ProcessY_UseExcessReturn"] = True
    params["ProcessX"] = True
    # params["ProcessX_Outlier"] = True
    # params["ProcessX_Normalization"] = True
    params["Symbols"] = data

    df = Analysis.LearningFramework.MultiPeriodData(database, datetime1, datetime2,
                                                    # factors=["Score","MScore"],
                                                    profileFolderName="MScoreFactor",
                                                    rangeReturnFolderName="Yearly",
                                                    params=params
                                                    )

    # df = pd.merge(df, dfSymbols, on="Symbol", how="inner")
    # print(df.shape[0])
    # df.drop(columns=["Symbol"], inplace=True)
    # print(df.head(20))
    # df.to_csv("d:/MScore_Factors.csv")

    # dfX = df[["GrossMargin_Index"]]
    dfX = df.drop(columns=["IsSignificant"])
    dfY = df[["IsSignificant"]]

    #
    Analysis.LearningFramework.Train(dfX, dfY)


def Strategy1(database):
    datetime1 = datetime.datetime(2007, 1, 1)
    datetime2 = datetime.datetime(2019, 6, 1)
    params = {}
    params["DropRedundantFields"] = True
    params["KeepFields"] = ["Symbol", "DateTime", "Return", "ExcessReturn"]
    params["ProcessY_Threshold"] = 0.2
    params["ProcessY_RiskAlpha"] = "Risk"
    params["ProcessY_UseExcessReturn"] = True
    params["ProcessX"] = True
    # params["ProcessX_Outlier"] = True
    # params["ProcessX_Normalization"] = True
    df = Analysis.LearningFramework.MultiPeriodData(database, datetime1, datetime2,
                                                    # factors=["Score","MScore"],
                                                    profileFolderName="MScoreFactor",
                                                    rangeReturnFolderName="Yearly",
                                                    params=params
                                                    )
    #
    coef = {}
    coef["Intercept"] = -4.84
    coef["DaysSaleRecv_Index"] = 0.92
    coef["GrossMargin_Index"] = 0.528
    coef["AssetQuality_Index"] = 0.404
    coef["Growth_CAGR_Sales_1Yr"] = 0.892
    coef["Depreciation_Index"] = 0.115
    coef["SGA_Index"] = -0.172
    coef["AccrualsToTotalAsset_LYR"] = 4.679
    coef["Leverage_Index"] = -0.327

    df["MScore"] = coef["Intercept"]\
                   + coef["DaysSaleRecv_Index"] * df["DaysSaleRecv_Index"] \
                   + coef["GrossMargin_Index"] * (1 / df["GrossMargin_Index"]) \
                   + coef["AssetQuality_Index"] * df["AssetQuality_Index"] \
                   + coef["Growth_CAGR_Sales_1Yr"] * df["Growth_CAGR_Sales_1Yr"] \
                   + coef["Depreciation_Index"] * (1 / df["Depreciation_Index"]) \
                   + coef["SGA_Index"] * df["SGA_Index"] \
                   + coef["AccrualsToTotalAsset_LYR"] * df["AccrualsToTotalAsset_LYR"] \
                   + coef["Leverage_Index"] * df["Leverage_Index"]

    df.to_csv("d:/MScore_Factors.csv")


def FactorToDataFrame(factorObjects):
    data = []
    for s in factorObjects:
        data.append([s["Symbol"], s["Value"]])
    dfFactor = pd.DataFrame(data, columns=["Symbol", "Value"])
    return dfFactor


def Strategy_SelectExtreme(database, datetime2):

    num = 100
    # datetime2 = datetime.datetime(2019, 5, 1)
    scores = database.Find("Factor", "MScore_Experience", {"DateTime": datetime2})
    dfScores = FactorToDataFrame(scores)
    #
    count = int(num*0.5)
    temp = dfScores.sort_values(by="Value", ascending=False)
    dfTop = temp[0:count]
    #
    temp = dfScores.sort_values(by="Value", ascending=True)
    dfBot = temp[0:count]
    #
    df = pd.concat([dfTop, dfBot], axis=0)
    print(dfTop.head(10))

    #
    path = "d://Data//Profile//MScore//"
    df.to_csv(path + "Portfolio_Extreme_" + Gadget.ToDateString(datetime2) + ".csv")
    IO.ExportWindTextFile_df(path + "Portfolio_Extreme_WindFile_" + Gadget.ToDateString(datetime2) + ".csv", df, datetime2)


def Strategy_Gap(database, datetime2):

    datetime1 = datetime.datetime(datetime2.year-1, 5, 1)
    #
    scores1 = database.Find("Factor", "MScore_Experience", {"DateTime": datetime1})
    dfScores1 = FactorToDataFrame(scores1)
    dfScores1["Value1"] = dfScores1["Value"]
    # print(dfScores1.head())
    dfScores1.drop(columns="Value", inplace=True)
    #
    scores2 = database.Find("Factor", "MScore_Experience", {"DateTime": datetime2})
    dfScores2 = FactorToDataFrame(scores2)
    dfScores2["Value2"] = dfScores2["Value"]
    dfScores2.drop(columns="Value", inplace=True)
    #
    df = pd.merge(dfScores1, dfScores2, how="inner", on="Symbol")
    df["Gap"] = df["Value2"] - df["Value1"]
    df.sort_values(by="Gap", ascending=True, inplace=True)
    df = df[0:100]
    #
    path = "d://Data//Profile//MScore//"
    df.to_csv(path + "Portfolio_PositiveGap_" + Gadget.ToDateString(datetime2) + ".csv")
    IO.ExportWindTextFile_df(path + "Portfolio_NegtiveGap_WindFile_" + Gadget.ToDateString(datetime2) + ".csv", df, datetime2)


if __name__ == '__main__':

    # ---Connecting Database---
    from Core.Config import *
    config = Config()
    database = config.DataBase("MySQL")

    # ---DateTime---
    datetime1 = datetime.datetime(2000, 1, 1)
    datetime2 = datetime.datetime(2019, 6, 1)

    #
    stocks = database.Find("Instruments", "Stock")
    indexs = database.Find("Instruments", "Index2")
    totalInstrument = stocks + indexs

    # 计算因子
    CalcMSCoreFactor(database, stocks, datetime1, datetime2, update=False)

    # 检查因子
    # CheckFactors(database)

    # 算分
    # datetime1 = datetime.datetime(2006, 12, 31)
    # RunModel(database, datetime1, None)

    # 输出到文件
    # ExportMScore(database)
    # ExportMScoreFactors(database)
    # SaveToDatabase(database)

    # TrainModel(database)

    # TestModel(database)

    # Strategy1(database)

    # Strategy_SelectExtreme(database, datetime.datetime(2017,5,1))
    # Strategy_SelectExtreme(database, datetime.datetime(2018,5,1))
    # Strategy_SelectExtreme(database, datetime.datetime(2019,5,1))

    Strategy_Gap(database, datetime.datetime(2017,5,1))
    Strategy_Gap(database, datetime.datetime(2018,5,1))
    Strategy_Gap(database, datetime.datetime(2019,5,1))